import requests,json,re
from lxml import etree
from datetime import datetime

data = []
def add(url='', i=0):
    url = f"https://www.hceb.edu.cn/xwzx/mtjm{url}.htm"
    response = requests.get(url)
    response.encoding = 'utf-8'
    h5 = etree.HTML(response.text)
    date_path = f'//*[@id="content"]/div[2]/ul/li[{i}]/cite/text()'
    span_path = f'//*[@id="content"]/div[2]/ul/li[{i}]/span/a/text()'
    uurl_path = f'//*[@id="content"]/div[2]/ul/li[{i}]/span/a/@href'
    date = h5.xpath(date_path)[0]
    span = h5.xpath(span_path)[0]
    uurl = h5.xpath(uurl_path)[0]
    title_re = re.compile(r'【([^】]*)】')  
    title = title_re.findall(span)[0] 
    body_re = re.compile(r'【[^】]*】(.*)')  
    body = body_re.findall(span)[0]  


    data_dict = {
        'date': date,
        'title':title,
        'body': body,
        'uurl': uurl
    }
    data.append(data_dict)


def spider():
    url = "https://www.hceb.edu.cn/xwzx/mtjm.htm"
    response = requests.get(url)
    # print(response.text)
    response.encoding = 'utf-8'
    h5 = etree.HTML(response.text)
    datalengiht_path = '//*[@id="content"]/div[3]/span[1]/text()'
    datalengiht = h5.xpath(datalengiht_path)[0]
    item = int(str(datalengiht).split('共')[1].split('条')[0])

    page = item // 20
    p = item % 20

    for items in range(page+1):
        if items==0:
            for item in range(1, 21):
                add('', item)
            continue
        if items == 1:
            for i in range(1, p + 1):
                add('/1', i)
            continue
        else:
            for item in range(1, 21):
                add('/'+str(page), item)

def getmtjmdata():
    global data
    spider()
    print(len(data), '条\n', data)
    try:
        with open('data.json', 'w', encoding='utf-8') as f:
            data1 = sorted(data, key=lambda item: datetime.strptime(item['date'], '%Y-%m-%d'),reverse=True)
            data = [{'id': idx + 1, **item} for idx, item in enumerate(data1)] 
            json.dump(data,f)
            print('写入成功！')
    except:
        print('写入失败！')

def get_mtjm():
    with open('data.json','r', encoding='utf-8') as f:
        data = json.load(f)
    return data
if __name__ == '__main__':
    getmtjmdata()
    # print(get_mtjm())